# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import re
from pymongo import MongoClient


class YangguangPipeline(object):
    def process_item(self, item, spider):
        item['content'] = self.demo(item['content'])
        # self.c.insert_one(item)
        print(item["content"])
        return item


    def open_spider(self, spider):
        print('spider runing------------')
        print('爬虫启动----------------')
        client = MongoClient()
        self.c = client['yangguang']['yg']

    def demo(self, data):
        data = [re.sub('\xa0|\s', '', i)  for i in data]
        data = [i for i in data if len(i) > 0]
        return data
